package com.app

import com.pro.{AdChannelAnalysis, AdDeviceTypeAnalysis, AdNetworkAnalysis, AdRegionAnalysis, AppAnaylysis, ProviceCityAnalysis}
import com.utils.ConfigUtils
import org.apache.kudu.spark.kudu.KuduContext
import org.apache.spark.sql.SparkSession

object Application {
  def main(args: Array[String]): Unit = {
    val spark: SparkSession = SparkSession.builder()
      .appName("app")
      .master("local[6]")
      .config("spark.sql.autoBroadcastJoinThreshold",ConfigUtils.SPARK_SQL_AUTOBROADCASTJOINTHRESHOLD)
      .config("spark.sql.shuffle.partitions",ConfigUtils.SPARK_SQL_SHUFFLE_PARTITIONS)
      .config("spark.shuffle.compress",ConfigUtils.SPARK_SHUFFLE_COMPRESS)
      .config("spark.shuffle.io.maxRetries",ConfigUtils.SPARK_SHUFFLE_IO_MAXRETRIES)
      .config("spark.shuffle.io.retryWait",ConfigUtils.SPARK_SHUFFLE_IO_RETRYWAIT)
      .config("spark.broadcast.compress",ConfigUtils.SPARK_BROADCAST_COMPRESS)
      .config("spark.serializer",ConfigUtils.SPARK_SERIALIZER)
      .config("spark.memory.fraction",ConfigUtils.SPARK_MEMORY_FRACTION)
      .config("spark.memory.storageFraction",ConfigUtils.SPARK_MEMORY_STORAGEFRACTION)
      .config("spark.default.parallelism",ConfigUtils.SPARK_DEFAULT_PARALLELISM)
      .config("spark.locality.wait",ConfigUtils.SPARK_LOCALITY_WAIT)
      .config("spark.speculation.multiplier",ConfigUtils.SPARK_SPECULATION_MULTIPLIER)
      .config("spark.speculation.flag",ConfigUtils.SPARK_SERIALIZER)
      .config("es.nodes",ConfigUtils.ES_NODES)
      .config("es.port",ConfigUtils.ES_PORT)
      .config("es.index.auto.create",ConfigUtils.ES_INDEX_AUTO_CREATE)
      .config("es.http.timeout",ConfigUtils.ES_HTTP_TIMEOUT)
      //.config("cluster.name","elasticsearch")
      .getOrCreate()
    spark.sparkContext.setLogLevel("warn")
    val kuduContext = new KuduContext(ConfigUtils.KUDU_MASTER,spark.sparkContext)
//    //todo 数据清洗
//    ParseIp.process(spark,kuduContext)
    //todo 省市地域统计
//    ProviceCityAnalysis.process(spark,kuduContext)
    //todo 地域分布情况
//    AdRegionAnalysis.process(spark,kuduContext)
    //todo app分布情况
//    AppAnaylysis.process(spark,kuduContext)
    //todo 手机设备类型分布情况
//    AdDeviceTypeAnalysis.process(spark,kuduContext)
    //todo 网络类型分布情况
//    AdNetworkAnalysis.process(spark,kuduContext)
    //todo 运营商分布情况统计
//    AdNetworkAnalysis.process(spark,kuduContext)
    //todo 渠道分布情况
    AdChannelAnalysis.process(spark,kuduContext)
  }

}
